
#---------------------------------------
#
#Script: svnlog.py
#Description: module to generate and parse svnlog files
import sys
import re
import _pg
import os
import repository
import popen2
import project
import pysvn
import datetime

#known extentions for known source file types
sourceExts = ['.h', '.c', '.cpp', '.java']

debugByLine = 0
#easy place to remove debug statments
def d(object):
    print object
    if(debugByLine != 0):
        str = raw_input("-")

class SVNRepository(repository.Repository):
    
    #TODO: add this method to the repository class and also to cvs
    def buildRepUsers(self, db, log):
        client = pysvn.Client()
        logEntries = client.log(self.path)
        
        authorSet = set([])
        for entry in logEntries:
            #if entry.author:
            authorSet.add(entry.author)
        for author in authorSet:
            sql = "INSERT INTO repusers (repname) values ('%s')" % author
            db.query(sql)

    def removeSlash(self, string):
        """remove first char of string if it is = '/'"""
        if string[0] == '/':
            return string[1:]
        else:
            return string

    def buildFiles(self, db, log):
        """Mine facts from SVN and store in the (already created) files table in the given database """
        client = pysvn.Client()
        logEntries = client.log(self.path, discover_changed_paths=True)
        #sort entries into an array by revision
        #find nescisary size
        max = 0
        for entry in logEntries:
            if entry.revision.kind == pysvn.opt_revision_kind.number:
                if max < entry.revision.number:
                    max = entry.revision.number
            else:
                #should probably do an exception instead
                print "non-number revision found. This is weird."
                sys.exit()

        #build a list of log entries sorted by revision and thus date
        entries = [None for i in range(max+1)]
        for entry in logEntries:
            entries[entry.revision.number] = entry
        

        for entry in entries:
            if entry == None:#the first item of the list should be None, and maybe there are missing revs or something
                continue

            d("Revision: " + str(entry.revision.number))
            for cp in entry.changed_paths:
                cp['path'] = self.removeSlash(cp['path'])#by convention remove first /, wash unicode
                cp['path'] = cp['path'].encode('ascii', 'backslashreplace')
                #debug info
                d(cp['path'])
                d("action: " +cp['action'])
                
                ###filter based on author 
                
                #anon commit
                if entry.author == '':
                   # d("no author, not committing")
                    pass
                #SVM commits are by Subversion Mirror script
                if entry.author == 'svm':
                    pass
                ###end filter

                ### Branch on the action performed on each file:
                
                #delete action
                if cp['action'] == "D":
                    d("not doing anything for a delete action")
                    continue
            
                #copied file. The action for these is usually A(dd) or R(elocate)
                #Fine the parent in the copyfrom data
                elif  cp['copyfrom_path']:
                    d("copy action")
                    parentPath = self.removeSlash(cp['copyfrom_path'])
                    parentRev = cp['copyfrom_revision']
                    if parentRev.kind != pysvn.opt_revision_kind.number:
                        print "non-number for revision of parent of file: %s" %cp['path']
                        sys.exit()
                    parentRev = parentRev.number
                    #We want to determine the parent file. This is not strait forward, you cannot just lookup
                    #the file and revision because the file may not have an entry at the exact revision, but instead
                    #some revision before it. Revisions are stored as strings so we can't just look for the closet
                    #revision either. We need to look for the most recent entry of that file by date, instead. 
                    q = "select fileid from files where filename = '%s' order by datetime desc limit 1" % parentPath
                    d(q)
                    pgo = db.query(q)
                    if pgo.getresult():
                        #if pgo.getresult():
                        d(pgo.getresult())
                        parentID = str(pgo.getresult()[0][0])
                        #else:
                        #    log.write("File not found in database, the svn seems to be inconsistant..." + parentPath + " rev: " + parentRev
                        #    continue
                    else:
                        parentID = "NULL"
                        #print "could not get parent id with name: %s and rev: %d (this should be a rare thing)" %(parentPath, parentRev)
                        #d(entry.message)
                        #asdf = raw_input("press enter to continue")
                        #sys.exit()
                
                #pure modify/add action
                elif cp['action'] in ['M', 'A']:
                    d("add or modify action")
                    #we need to determine the parent file 
                    #lets try the last one in the database
                    q = "select fileid from files where filename = '%s' order by datetime desc limit 1" % cp['path']
                    d(q)
                    pgo = db.query(q)
                    result = pgo.getresult()
                    d(result)
                    if result:
                        parentID = str(result[0][0])
                    else:
                        parentID = "NULL"
                        #print "could not get parent id with name: %s, filename not in SQL yet" %(parentPath)
                        #sys.exit()
                #unknown action!
                else:
                    print "action: %s msg: %s" % (entry.action, entry.message)
                    print "unknown file action. this just means svn.py isn't done yet, and you should add functionality to handle this type of entry"
                    sys.exit()
                
                ### END BRANCH

                #determine if file is source
                if "." + cp['path'].split(".")[-1] in sourceExts:
                    isSource = True
                else:
                    isSource = False
                
                #discover if its a dir(tag)
                #for k in entry.changed_paths:
                #    print k.values()
                commitq = """INSERT INTO files (Datetime, RepID, FileName, Revision, sourcefile, parent)
                VALUES('%s', (select RepID from RepUsers where RepName = '%s'),'%s',%s,%s,%s)
                """ % (datetime.datetime.fromtimestamp(entry.date).isoformat(), entry.author, cp['path'], str(entry.revision.number), str(isSource), parentID)
                
                d(commitq)
                try:
                    db.query(commitq)
                except _pg.ProgrammingError, e:
                    log.write("Exception! could not commit: %s because: %s" % (commitq, str(e)) )
                    raise e
    
    def removeDirectories(db, log):
        """remove file entries which are directories from the files table. Assumes filetable is complete"""
        #this will select most of the directories, but potentially could have false positives
        #if someone had file "foo" then deleted it to make directory "foo" in the same place. 
        #if someone works on this function, you should just delete those lines inside of SQL with a longer statement
        #and then next check every unique filename which does not have a file extension using the 'ls' command
        #on the file in the working copy
        q = "select distinct files1.filename from files files1, files files2 where length(files2.filename) > length(files1.filename) and strpos(files2.filename, files1.filename) = 1"
           

        

    #write the svn log to a file
    def createLog(self, svnLogFileName):
        #change to svn dir, saving cwd
        print "creating svn log..."
        scriptPath = os.getcwd()
        os.chdir(self.path)
        s = 'svn log -v > ' + svnLogFileName
        print "command is", s
        os.system(s)
        #go back to where we ran script from            
        os.chdir(scriptPath)


   
    def prepUpdate(self):
        """get the WC ready for mining"""
        popen2.popen4("svn cleanup")
 
    def updateRepoToRev(self, revision):
        """Update whole repo to rev"""
        updateCmd = "cd %s && svn update -r %s " % (self.path, revision)
        d("in" + self.path)
        result = popen2.popen4(updateCmd)[0].read()
        d( updateCmd +  result)

    def updateFileToRev(self, revision):
        """Update the file in the WC to the given revision"""
        updateCmd = "cd %s && svn update -r %s %s" % (self.path, revision, filename)
        print "in", self.path
        result = popen2.popen4(updateCmd)[0].read()
        print updateCmd, result

    def updateToDate(self, date):
        """takes in numerical date and year and updates the svn working copy"""
        updateCmd = "cd %s && svn update -r {%s}" % (self.path, str(date))
        print "in", self.path
        result = popen2.popen4(updateCmd)[0].read()
        print updateCmd, result
    
    def isDir(self, filename, rev):
        """returns true if filename:rev is a dir in the svn"""
        #if you want to check the status of a file inside of a working copy, the file must be updated
        #to the revision you are looking at. The other option is to directly check the SVN reposity
        #with its URL.
        #we're doing the first method, since we don't have the URL handy:
        
        client = pysvn.Client()
       # fileurl = self.URL + '/' + filename
        revobj = pysvn.Revision(pysvn.opt_revision_kind.number, rev)
        print "updating file:", self.path+filename, "to revision:", rev
        client.update(self.path, revision=revobj)
        # 
       # entries = client.ls(fileurl, revision=revobj, recurse=False)
       # print entries
       # s = raw_input(">")
       # if len(entries) == 1:
       #     entry = re.sub('/[^/]*$', '', entries[0]['name'])
       #     if entry == fileurl and entries[0]['kind'] == pysvn.node_kind.file:
       #         print "FOUND FILE:", filename, "with node:", entries[0]['kind']
       #         return False
       # return True
        
        s = "svn ls -r%s %s" % (rev, self.path + filename)
        print s
        list = popen2.popen4(s)[0].read()
        print list
        list = list.split()
        print len(list)   
        if len(list) > 1:#must be dira
            print "must be a populated dir"
            return True
        if len(list) == 0:#must be empty dir
            print "must be a empty dir"
            return True
        length = len(list[0])
        if filename[-length:]==list[0]: #ls file returns "file"
            print "File!"
            return False
        print "dir by default"
        return True

    def mineLOC(self, db, log):
        """function to mine CLOC, SLOC from source"""
        #pull down sql data on source files
        s = 'select filename, revision from files where sourcefile=true order by revision'
        data = db.query(s).getresult()
        lastRev = 0
        client = pysvn.Client()
        for row in data:
            revision = row[1]
            filename = row[0]
            d(row)
            if lastRev != revision:
                #we have to update the whole repo just to get at the filename we're interested in, because if that file doesn't exist yet, it can't be targetd for an update. 

                #Falls to unicode error: client.update(self.path, revision=pysvn.Revision(pysvn.opt_revision_kind.number, revision))
                self.updateRepoToRev(revision)
                lastRev = revision
                d("updated to revision: %s" %lastRev)
            command = "cd %s && kloc -i %s" %(self.path, filename)
            d("trying command:" + command)
            while 1:
                try:
                    stdout, stdin = popen2.popen4(command)
                    break
                except OSError, e:
                    print e
                    print "trying again..."
                
            result = stdout.readlines()
            stdout.close()
            stdin.close()
            q = "update files set codelines=%s, commentlines=%s, inlinecommentlines=%s where filename='%s' and revision='%s'" % (
                result[0], result[1], result[2], row[0], row[1])
            #commentlines is pure comment lines
            #codelines is pure code lines
            #inlinecommentlines is both code and comment lines
            try:
                db.query(q)
            except Exception, e:
                print "error! kloc output:", result
                raise e
        #checkout those files and run kloc on them, do SQL update
    
    
    def getFileDict(self, db):
        "load the files + revisions from sql, return data in a hash RepFiles keyed by filename"
        
        pgo = db.query("select Filename, Revision from files where sourcefile = True ORDER BY datetime");
        #stores data about what files we haven't mined data on yet
        fileDataHash = {}
        if(pgo != None):
            results = pgo.getresult()
            for filename, revision in results:
                if(fileDataHash.has_key(filename)):
                    fileDataHash[filename].append(revision)
                else:
                    fileDataHash[filename] = [revision]
        else:
            print "Could not get commit data from database!"
            return None

        return fileDataHash

    def updateRepository(self):
        s = "cd %s && svn up"%self.path
        popen2.popen4(s)

  
    #OUT DATED - ancestry is mined during the first pass now!
    def mineAncestry(self, db, log):
        """Mine out the ancestry between file revisions, and store in SQL"""
        import xml.dom.minidom
        #get all distinct files
        fileDict = self.getFileDict(db)
        
        for file, revisions in fileDict.items():
            ancestry = []
            iRevisions = map(int, revisions)
            iRevisions.sort()
            
            #basically we're trying to make sure every revision of a filename is accounted for in the svn log. If a filename/path was used twice over the repositories lifetime, their will still be revisions left in iRevisions after the first pass. 
            while len(iRevisions) > 0:
                rev = iRevisions[-1]#most recent revision we havn't accounted for
                self.updateFileToRev(file, str(rev))#must update to the version we want the log of
                xmlData = self.getFileLog(file)
                rootNode = xml.dom.minidom.parseString(xmlData)
                for entry in rootNode.childNodes[0].childNodes:
                    if entry.nodeName == 'logentry':
                        #get the revision of this log entry
                        entryRev = int(entry.attributes.item(0).nodeValue)
                        ancestry.append(entryRev)
                        iRevisions.remove(entryRev)

                print file
                print iRevisions
                print ancestry
                #use ancestry to create ancestry tree in SQL
                for ii in range(len(ancestry) - 1): #loop over n-1 ancestry entries
                    #ii is the ancestor of ii+1
                    q = "update files set "         
               
if __name__ == "__main__":
    parseSVNLog(open(sys.argv[1]))
    
